from PyQt4 import QtCore
from PyQt4.QtCore import QObject,QThread
from KDataBase.KDBProvider import KAbstractDBProvider
from KDataBase.KDBUtility import KDBUtility
import time
import os ;
import re ;
import datetime
from datetime import timedelta
#from pytz import timezone
#import pytz
class KStooqProvider(KAbstractDBProvider.KAbstractDBProvider):
	def __init__(self,parent,database):
		super(KStooqProvider,self).__init__(parent,database)
		self.premarket="" ## record previous considered market
	'''Import related functions'''

	def BuildAllHistory(self,path):
		## scan all market under current path,the hieracy of Stooq data should be stooq--country--market--symbol      
		self.stopped=False   ## stop running flag  
		self.workingMethod=self.__BuildHistory
		self.start=time.clock()
		finished=self.__ScanAllStooqFolderInternal(path,0,"","")  ##serach recursively
		self.UpdateProgress("Finished Building All History")
		return finished 
	def __ScanAllStooqFolderInternal(self,path,level,country,market):
	##build all country.market data internally
		if level==1:
			country=path.split("\\")[-1]
		elif level>1:
			market=market+("." if market!="" else "")+path.split("\\")[-1]   
		subfolders = os.listdir(path)
		for country_dir in subfolders: ## each country
			abspath=os.path.join(path, country_dir)
			if os.path.isdir(abspath)==False:
				continue
			subdirlist=[o for o in os.listdir(abspath) if os.path.isdir(os.path.join(abspath, o) )]##has subdirectory?
			if len(subdirlist)>0:
				self.__ScanAllStooqFolderInternal(abspath,level+1,country,market)
			else:
				tempmarkt=abspath.split("\\")[-1]  
				if not tempmarkt.isdigit():##do not to append market string if is number
					curmarket=market+("." if market!=""  else "")+tempmarkt
				else:
					curmarket=market
				self.UpdateProgress("Parsing Country:%s, Market:%s  " %(country,curmarket))
				print(abspath)		
				finished=self.workingMethod(abspath,country,curmarket) ## scan market directory
				finished=True
				self.premarket=curmarket ## update previous searched market.
				if finished==False: ## return because interrupted by user.
					return False                  


	def __BuildHistory(self,folder,country,market):
		## Build current market from EOD data provider
		subfolders = os.listdir(folder)
		##self.db.drop_collection(self.GetCollectionName(country,market,False))
		self.cldailystocks=self.db[KDBUtility.GetCollectionName(country,market,False)]
		datacount=0
		upcontents=[]
		if market.upper() !='NASDAQ':
			pass
		for file in subfolders:#scan for each symbol
			symbol=file.split('.',2)[0].upper().replace('_','/') # stooq already change '.' to '_'
			asbfilepath=os.path.join(folder,file)
			data=open(os.path.join(folder,file))
			if os.stat(asbfilepath)[6]==0: ## skip empty file
				continue;
			next(data)#skip first line
			for each_line in data: #scan for each date
				(sdate,openp,highp,lowp,closep,vol,inst)=each_line.split(',',7)
				sdate=sdate.lstrip('0') # remove noise zero
				priceblock=[openp,highp,lowp,closep,vol]
				newcondition={"symbol":symbol,"date":sdate}
				upcontents+=[{"symbol":symbol,"date":sdate,"price":priceblock,"provider":'stooq'}]
				datacount+=1
				if datacount%self.bulklimit==0:
					self.BulkInsertBySingleStockDate(market,datacount,upcontents)
					upcontents=[]			
			data.close()   
			if self.Stop: 
				return False
		if upcontents:
			self.BulkInsertBySingleStockDate(market,datacount,upcontents)
		self.UpdateProgress("Ensure Index for M:%s" %(market))
		self.cldailystocks.ensure_index([("symbol",1)]) ## build index
		return True

	''' update related functions'''

	def UpdateAllHistory(self,path,timerange):
		## update database based on an incremetal basis,the hieracy of Stooq data should be stooq--country--market--symbol
		self.stopped=False   ## stop running flag  
		self.workingMethod=self.__UpdateHistory
		self.premarket=""
		self.updatedata=[]
		self.start=time.clock()
		self.__ScanAllStooqFolderInternal(path,0,"","")  ##serach recursively
		finished=self.UpdateProgress("Finished Updating Stooq History")  
		return finished        
	def __UpdateHistory(self,folder,country,market):
		## Build current market from EOD data provider
		subfolders = os.listdir(folder)
		##self.db.drop_collection(self.GetCollectionName(country,market,False))
		self.cldailystocks=self.db[KDBUtility.GetCollectionName(country,market,True)]

		if self.cldailystocks.count()==0:
			self.UpdateProgress("Datbase not exisit for country: %s market %s"%(country,market))  
			return True
		stockcount=0
		if self.premarket!=market: ## clear data dict
			self.updatedata={}
		dailyfoldr=KDBUtility.GetDailyBackupPath("Daily",country,market)
		minfolder=KDBUtility.GetDailyBackupPath("5Min",country,market)
		for file in subfolders:#scan for each symbol
			symbol=file.split('.',2)[0].replace('_','/') # stooq already change '.' to '_'
			asbfilepath=os.path.join(folder,file)
			data=open(os.path.join(folder,file))
			if os.stat(asbfilepath)[6]==0: ## skip empty file
				continue;
			next(data)#skip first line
			for each_line in data: #scan for each date
				(sdate,stime,openp,highp,lowp,closep,vol,inst)=each_line.split(',',7)
				sdate=sdate.lstrip('0') # remove noise zero
				priceblock=[float(openp),float(highp),float(lowp),float(closep),int(vol)]
				sdate=datetime.datetime(int(sdate[0:4]), int(sdate[5:7]), int(sdate[8:10]),int(stime[0:2]),int(stime[3:5]),int(stime[6:8]))
				sdate=sdate-timedelta(hours=6) # timezone to stooq local
				sdatekey=int(sdate.strftime("%Y%m%d"))
				if sdatekey not in self.updatedata:
					self.updatedata[sdatekey]={}
				curdatedic=self.updatedata[sdatekey]
				if symbol not in curdatedic:
					curdatedic[symbol]={k:[] for k in ('time','price')}
					cursymboldic=curdatedic[symbol]                    
				cursymboldic=curdatedic[symbol]
				cursymboldic['time'].append(int(sdate.strftime("%H%M")))
				cursymboldic['price'].append(priceblock)            
			data.close()
			if self.Stop: 
				return False
			stockcount+=1
			if stockcount%self.stockcountlimit==0:## warning: udpate too fast will freeze the user interface.
				self.UpdateProgress("scaned count: %d in M: %s " %(stockcount,market))
				#break;#for test
		self.UpdateSingleStock(self.updatedata,(dailyfoldr,minfolder),self.cldailystocks)   
		self.UpdateProgress("Finished M: %s Total count: %d)" %(market,stockcount))
		return True